#coding:utf-8
import numpy as np
import pandas as pd

class Data_Process:
    data_csv_file='../../doc/data/LoanStats3a.csv'
    data_feature_file='../../doc/data/feature0a.csv'
    
    def read_process_data(self):
        df=pd.read_csv(self.data_csv_file,skiprows=1,low_memory=False)
        df.drop('id',1,inplace=True)
        df.drop('member_id',1,inplace=True)
        df.term.replace(to_replace='[^0-9]+',value='',inplace=True,regex=True)
        df.term.astype(float)
        #df.int_rate.replace('%','',inplace=True)
        df.int_rate.replace(to_replace='%',value='',inplace=True,regex=True)
        df.drop(['sub_grade','emp_title'],1,inplace=True)
        df.emp_length.replace('n/a',np.nan,inplace=True)
        df.emp_length.replace(to_replace='[^0-9]+',value='',inplace=True,regex=True)
       
        df.dropna(axis=1, how='all', inplace=True)
        df.dropna(axis=0, how='all', inplace=True)

        df.drop(['mths_since_last_record','next_pymnt_d','debt_settlement_flag_date','settlement_status',\
                 'settlement_date','settlement_amount','settlement_percentage','settlement_term'],\
                1,inplace=True)
        #统计float类型
        for col in df.select_dtypes(include=['float']).columns:
            print('col {} has {}'.format(col, len(df[col].unique())))
        #删除float中分类与实例个数不在一个数量级上的
        df.drop(['collections_12_mths_ex_med','policy_code','acc_now_delinq','chargeoff_within_12_mths',\
                 'delinq_amnt','pub_rec_bankruptcies','tax_liens','total_acc','out_prncp','out_prncp_inv',\
                 'delinq_2yrs','inq_last_6mths','mths_since_last_delinq','open_acc','pub_rec'],1,inplace=True)
        #查看object类型中占比很少的数据
        for col in df.select_dtypes(include=['object']).columns:
            print('col {} has {}'.format(col,len(df[col].unique())))
        ##删除object类型数据    
        df.drop(['term','int_rate','grade','emp_length','home_ownership','verification_status','issue_d',\
                 'desc','pymnt_plan','purpose','zip_code','addr_state','earliest_cr_line','initial_list_status',\
                 'last_pymnt_d','last_credit_pull_d','application_type','hardship_flag','disbursement_method',\
                 'debt_settlement_flag'],1,inplace=True)
        #查看y状态，并删除y为空的实例(按行删除)
        df.loan_status.replace('Fully Paid',int(1),inplace=True)
        df.loan_status.replace('Charged Off',int(0),inplace=True)
        df.loan_status.replace('Does not meet the credit policy. Status:Fully Paid',np.nan,inplace=True)
        df.loan_status.replace('Does not meet the credit policy. Status:Charged Off',np.nan,inplace=True)
        df.dropna(subset=['loan_status'],inplace=True)
        df.drop('title',1,inplace=True)
        df.fillna(0,inplace=True)
        df.fillna(0.0,inplace=True)
        #删除相关性太高的列
        df.drop(['loan_amnt','funded_amnt','total_pymnt'],1,inplace=True)
        #正交化处理，删除相关性高的列
        cor=df.corr() #pandas 求协方差矩阵公式
        cor.loc[:,:]=np.tril(cor, k=-1)
        cor=cor.stack() #结构重建
        print(cor[(cor>0.55)|(cor<-0.55)])
        # 进行哑变量处理
        df=pd.get_dummies(df)
        df.to_csv(self.data_feature_file)
        
        